import cv2
import os
import numpy as np
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing import image
from PIL import Image
import tensorflow as trf
from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Input, BatchNormalization
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix, classification_report
from keras.optimizers import Adam
import warnings
warnings.filterwarnings('ignore')
dataset_folder = r'C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val'
classes = ['COVID-19', 'Non-COVID', 'Normal']
Checking for the balance of images is important because an imbalanced dataset can bias the model towards the majority class and lead to overfitting. So, let's check the balance of images.
def plot_class_distribution(data_dirs, xlabel_dict):
for data_dir in data_dirs:
class_counts = {}
total_samples = 0
class_name = os.path.basename(data_dir)
x_labels = xlabel_dict.get(class_name, ["Images", "Lung Masks"])
for subdir, xlabel in zip(["Images", "Lung Masks"], x_labels):
subdir_path = os.path.join(data_dir, subdir)
num_samples = len(os.listdir(subdir_path))
class_counts[xlabel] = num_samples
total_samples += num_samples
plt.figure(figsize=(9, 6))
plt.bar(class_counts.keys(), class_counts.values())
plt.xlabel('Subdirectory')
plt.ylabel('Number of Samples')
plt.title(f'Class Distribution for {class_name}')
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
plt.show()
print(f"Total number of samples for {class_name}: {total_samples}")
data_directories = [
r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\COVID-19",
r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Normal",
r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Non-COVID"
]
xlabel_dict = {
"COVID-19": ["covid Images", "Covid Lung Masks"],
"Normal": ["nrmal Images", "Normal Lung Masks"],
"Non-COVID": ["non-Covid Images", "Non-Covid Lung Masks"]
}
plot_class_distribution(data_directories, xlabel_dict)
Total number of samples for COVID-19: 3806
Total number of samples for Normal: 3424
Total number of samples for Non-COVID: 3604
We don't have imbalanced data (images). There's a difference of around 100 images. Therefore, there is no need to balance the images.
image_path = r"C:\Users\kruta\Downloads\Deep learning\Case study 1\CovidXRayImages\Val\Non-COVID\images\non_COVID (6143).png" # Replace with the path to your image file
image = Image.open(image_path)
image_size = image.size
print("Image size:", image_size)
Image size: (256, 256)
Since original images are 256*256 pixels, we could consider resizing them to a commonly used input size such as (224, 224) or (299, 299)
As the lung is located in the middle of the image, we can consider cropping the image.
Normalization is like putting all our data on the same scale to make it fair and balanced. It ensures consistency, allows for fair comparison, and improves performance in data analysis. By converting measurements to a common scale, normalization helps us analyze data more effectively and draw meaningful insights from it.
#target size for resizing
target_size = (224, 224)
#performing MinMaxScaler
scaler = MinMaxScaler()
normalized_images = []
normalized_masks = []
labels = []
#load and resizing
def load_and_resize_image(image_path, target_size):
img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
resized_img = cv2.resize(img, target_size)
return resized_img
# Loop through each class folder
for class_index, class_name in enumerate(classes):
image_folder = os.path.join(dataset_folder, class_name, 'images')
mask_folder = os.path.join(dataset_folder, class_name, 'lung masks')
for filename in os.listdir(image_folder):
image_path = os.path.join(image_folder, filename)
mask_path = os.path.join(mask_folder, filename)
# Load and resize images
original_img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
resized_img = load_and_resize_image(image_path, target_size)
# Normalize the resized image
normalized_img = scaler.fit_transform(resized_img.astype(float).reshape(-1, 1)).reshape(resized_img.shape)
# Load and resize masks
mask_img = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE).astype(np.uint8)
resized_mask = load_and_resize_image(mask_path, target_size)
# Normalizinge resized mask
normalized_mask = scaler.fit_transform(resized_mask.astype(float).reshape(-1, 1)).reshape(resized_mask.shape)
# Append
normalized_images.append(normalized_img)
normalized_masks.append(normalized_mask)
labels.append(class_index)
#lists to arrays
normalized_images = np.array(normalized_images)
normalized_masks = np.array(normalized_masks)
labels = np.array(labels)
print("Img_Size:", normalized_images.shape)
print("Masks_Size:", normalized_masks.shape)
Img_Size: (5417, 224, 224) Masks_Size: (5417, 224, 224)
ori_pixel = original_img.flatten()
norm_pixel = normalized_img.flatten()
# Display original, resized, and normalized images
for i in range(7):
plt.figure(figsize=(15, 6))
plt.subplot(1, 4, 1)
plt.imshow(original_img, cmap='gray')
plt.title('Original Image')
plt.subplot(1, 4, 2)
plt.imshow(resized_img, cmap='gray')
plt.title('Resized Image')
plt.subplot(1, 4, 3)
plt.imshow(normalized_img, cmap='gray')
plt.title('Normalized Image')
#before and after normalization
print("Pixel values before normalization:")
print(ori_pixel)
# print()
print("\nPixel values after normalization:")
print(norm_pixel)
print()
Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036] Pixel values before normalization: [160 121 99 ... 70 58 35] Pixel values after normalization: [0.61354582 0.4501992 0.3625498 ... 0.30278884 0.24302789 0.14741036]
There is no need to normalize black and white images since they already range between 0 and 1. However, let's compare the values to confirm.
ori_pixel1 = mask_img.flatten()
norm_pixel1 = normalized_mask.flatten()
#plotting for masks
for i in range(5):
plt.figure(figsize=(15, 6))
plt.subplot(1, 4, 1)
plt.imshow(mask_img, cmap='gray')
plt.title('Original Image')
plt.subplot(1, 4, 2)
plt.imshow(resized_mask, cmap='gray')
plt.title('Resized Image')
plt.subplot(1, 4, 3)
plt.imshow(normalized_mask, cmap='gray')
plt.title('Normalized Image')
#before and after normalization
print("Pixel values before normalization:")
print(ori_pixel1)
print("\nPixel values after normalization:")
print(norm_pixel1)
Pixel values before normalization: [0 0 0 ... 0 0 0] Pixel values after normalization: [0. 0. 0. ... 0. 0. 0.] Pixel values before normalization: [0 0 0 ... 0 0 0] Pixel values after normalization: [0. 0. 0. ... 0. 0. 0.] Pixel values before normalization: [0 0 0 ... 0 0 0] Pixel values after normalization: [0. 0. 0. ... 0. 0. 0.] Pixel values before normalization: [0 0 0 ... 0 0 0] Pixel values after normalization: [0. 0. 0. ... 0. 0. 0.] Pixel values before normalization: [0 0 0 ... 0 0 0] Pixel values after normalization: [0. 0. 0. ... 0. 0. 0.]
We can also perform noise reduction techniques for preprocesssing such as,
(1). Gaussian Blur: This method uses a special filter called a Gaussian filter to gently soften the image, making it smoother and reducing small details. It's great for getting rid of the grainy look in pictures, especially in areas with lots of fine details.
(2). Median Filtering: Instead of averaging neighboring pixel values like many filters, median filtering takes the middle value among the neighboring pixels. This is especially useful for cleaning up images with random specks or dots (like salt-and-pepper noise) while keeping the edges of objects nice and sharp.
reference: https://medium.com/@anishaswain/noise-filtering-in-digital-image-processing-d12b5266847c
# Combine preprocessed(resized and normalized) images and masks
combined_data = list(zip(normalized_images, normalized_masks,labels))
train_data, test_data = train_test_split(combined_data, test_size=0.2, random_state=42)
# Unpack the data into separate lists for images, masks, and labels
def unpack_data(data):
if data:
return zip(*data)
else:
return [], [], []
# Unpack training data
X_train, masks_train, y_train = unpack_data(train_data)
X_test, masks_test, y_test = unpack_data(test_data)
X_train = np.array(X_train)
y_train = np.array(y_train)
masks_train = np.array(masks_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
masks_test = np.array(masks_test)
y_train
array([2, 1, 2, ..., 2, 2, 0])
Max pooling selects the brighter pixels from the image. It is useful when the background of the image is dark and we are interested in only the lighter pixels of the image.
Here we have four approaches, but we're going with the averaging approach.
input_shape_image = (224, 224, 1)
input_shape_mask = (224, 224, 1)
# CNN model for image data
model_image = Sequential([
Conv2D(32, (3, 3), activation='relu', input_shape=input_shape_image),
MaxPooling2D((2, 2)),
Conv2D(64, (3, 3), activation='relu'),
MaxPooling2D((2, 2)),
Conv2D(64, (3, 3), activation='relu'),
Flatten(),
Dense(64, activation='relu'),
Dense(3, activation='softmax')
])
model_image.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 222, 222, 32) 320
max_pooling2d (MaxPooling2D (None, 111, 111, 32) 0
)
conv2d_1 (Conv2D) (None, 109, 109, 64) 18496
max_pooling2d_1 (MaxPooling (None, 54, 54, 64) 0
2D)
conv2d_2 (Conv2D) (None, 52, 52, 64) 36928
flatten (Flatten) (None, 173056) 0
dense (Dense) (None, 64) 11075648
dense_1 (Dense) (None, 3) 195
=================================================================
Total params: 11,131,587
Trainable params: 11,131,587
Non-trainable params: 0
_________________________________________________________________
model_image.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
# Convert labels to one-hot encoded vectors
y_train_one_hot = to_categorical(y_train, num_classes=3)
y_test_one_hot = to_categorical(y_test, num_classes=3)
# Train Image Model
model_image.fit(X_train, y_train_one_hot, epochs=7, batch_size=30)
Epoch 1/7 145/145 [==============================] - 109s 738ms/step - loss: 1.0040 - accuracy: 0.5751 Epoch 2/7 145/145 [==============================] - 106s 730ms/step - loss: 0.6508 - accuracy: 0.7390 Epoch 3/7 145/145 [==============================] - 107s 735ms/step - loss: 0.5265 - accuracy: 0.7914 Epoch 4/7 145/145 [==============================] - 104s 715ms/step - loss: 0.4110 - accuracy: 0.8401 Epoch 5/7 145/145 [==============================] - 103s 707ms/step - loss: 0.2999 - accuracy: 0.8800 Epoch 6/7 145/145 [==============================] - 105s 721ms/step - loss: 0.1942 - accuracy: 0.9301 Epoch 7/7 145/145 [==============================] - 102s 704ms/step - loss: 0.1306 - accuracy: 0.9541
<keras.callbacks.History at 0x2552f9b6b80>
# Evaluate the image model on test data
image_loss, image_accuracy = model_image.evaluate(X_test, y_test_one_hot)
print("Image Model - Test Loss:", image_loss)
print("Image Model - Test Acc:", image_accuracy)
34/34 [==============================] - 5s 131ms/step - loss: 0.7415 - accuracy: 0.7906 Image Model - Test Loss: 0.7414774894714355 Image Model - Test Acc: 0.7905904054641724
# CNN model for mask data
model_mask = Sequential([
Conv2D(32, (3, 3), activation='relu', input_shape=input_shape_mask),
MaxPooling2D((2, 2)),
Conv2D(64, (3, 3), activation='relu'),
MaxPooling2D((2, 2)),
Conv2D(64, (3, 3), activation='relu'),
Flatten(),
Dense(64, activation='relu'),
Dense(3, activation='softmax')
])
model_mask.summary()
Model: "sequential_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_3 (Conv2D) (None, 222, 222, 32) 320
max_pooling2d_2 (MaxPooling (None, 111, 111, 32) 0
2D)
conv2d_4 (Conv2D) (None, 109, 109, 64) 18496
max_pooling2d_3 (MaxPooling (None, 54, 54, 64) 0
2D)
conv2d_5 (Conv2D) (None, 52, 52, 64) 36928
flatten_1 (Flatten) (None, 173056) 0
dense_2 (Dense) (None, 64) 11075648
dense_3 (Dense) (None, 3) 195
=================================================================
Total params: 11,131,587
Trainable params: 11,131,587
Non-trainable params: 0
_________________________________________________________________
# Compile the mask model
model_mask.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
# Train Mask Model
model_mask.fit(masks_train, y_train_one_hot, epochs=7, batch_size=30)
Epoch 1/7 145/145 [==============================] - 105s 720ms/step - loss: 1.0780 - accuracy: 0.4690 Epoch 2/7 145/145 [==============================] - 100s 689ms/step - loss: 0.8719 - accuracy: 0.5844 Epoch 3/7 145/145 [==============================] - 99s 684ms/step - loss: 0.7338 - accuracy: 0.6681 Epoch 4/7 145/145 [==============================] - 98s 677ms/step - loss: 0.5077 - accuracy: 0.7895 Epoch 5/7 145/145 [==============================] - 102s 704ms/step - loss: 0.2559 - accuracy: 0.9045 Epoch 6/7 145/145 [==============================] - 97s 668ms/step - loss: 0.1084 - accuracy: 0.9603 Epoch 7/7 145/145 [==============================] - 97s 669ms/step - loss: 0.0520 - accuracy: 0.9843
<keras.callbacks.History at 0x2573cbd8640>
# Evaluate the mask model on test data
mask_loss, mask_accuracy = model_image.evaluate(masks_test,y_test_one_hot)
print("Mask Model - Test Loss:", mask_loss)
print("Mak Model - Test Accuracy:", mask_accuracy)
34/34 [==============================] - 4s 119ms/step - loss: 21.7385 - accuracy: 0.4077 Mask Model - Test Loss: 21.73845863342285 Mak Model - Test Accuracy: 0.40774908661842346
here, we can also take weighted average if we want to give more weights to either image or mask.
# Combine predictions by taking avegae
combined_pred = ((model_image.predict(X_test) + model_mask.predict(masks_test))/2)
34/34 [==============================] - 4s 127ms/step 34/34 [==============================] - 4s 121ms/step
auc_score = roc_auc_score(y_test_one_hot, combined_pred)
print("Combined AUC_score:", auc_score)
Combined AUC_score: 0.8898847877182926
# Convert predictions to class
combinedlabels = np.argmax(combined_pred, axis=1)
truelabels = np.argmax(y_test_one_hot, axis=1)
# Confusion Matrix
confmatrix = confusion_matrix(truelabels, combinedlabels, labels=np.arange(len(classes)))
plt.figure(figsize=(8, 6))
sns.heatmap(confmatrix, annot=True, fmt='d', cmap='Blues', xticklabels=classes, yticklabels=classes)
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()
# classification report
class_report = classification_report(truelabels, combinedlabels, target_names=classes)
print(class_report)
precision recall f1-score support
COVID-19 0.72 0.73 0.73 406
Non-COVID 0.73 0.72 0.73 328
Normal 0.72 0.71 0.72 350
accuracy 0.72 1084
macro avg 0.72 0.72 0.72 1084
weighted avg 0.72 0.72 0.72 1084
input_shape_image = (224, 224, 1)
input_shape_mask = (224, 224, 1)
# Define the DNN model for image data
model_image = Sequential([
Flatten(input_shape=input_shape_image),
Dense(128, activation='relu'),
Dense(64, activation='relu'),
Dense(3, activation='softmax')
])
# Compile the image model
model_image.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model_image.summary()
Model: "sequential_2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
flatten_2 (Flatten) (None, 50176) 0
dense_4 (Dense) (None, 128) 6422656
dense_5 (Dense) (None, 64) 8256
dense_6 (Dense) (None, 3) 195
=================================================================
Total params: 6,431,107
Trainable params: 6,431,107
Non-trainable params: 0
_________________________________________________________________
# Train the image model
model_image.fit(X_train, y_train_one_hot, epochs=7, batch_size=30)
Epoch 1/7 145/145 [==============================] - 10s 69ms/step - loss: 3.0698 - accuracy: 0.4853 Epoch 2/7 145/145 [==============================] - 11s 72ms/step - loss: 1.0989 - accuracy: 0.5800 Epoch 3/7 145/145 [==============================] - 10s 66ms/step - loss: 0.8901 - accuracy: 0.6227 Epoch 4/7 145/145 [==============================] - 9s 64ms/step - loss: 0.9238 - accuracy: 0.6324 Epoch 5/7 145/145 [==============================] - 9s 64ms/step - loss: 0.9694 - accuracy: 0.6185 Epoch 6/7 145/145 [==============================] - 9s 63ms/step - loss: 0.8375 - accuracy: 0.6358 Epoch 7/7 145/145 [==============================] - 9s 64ms/step - loss: 0.7964 - accuracy: 0.6580
<keras.callbacks.History at 0x2573ceeb760>
# Evaluate the image model on test data
image_loss, image_accuracy = model_image.evaluate(X_test, y_test_one_hot)
print("Image Model - Test Loss:", image_loss)
print("Image Model - Test Acc:", image_accuracy)
34/34 [==============================] - 0s 5ms/step - loss: 1.0943 - accuracy: 0.5498 Image Model - Test Loss: 1.0942562818527222 Image Model - Test Acc: 0.5498154759407043
# Define the DNN model for mask data
model_mask = Sequential([
Flatten(input_shape=input_shape_mask),
Dense(128, activation='relu'),
Dense(64, activation='relu'),
Dense(3, activation='softmax')
])
model_mask.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model_mask.summary()
Model: "sequential_3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
flatten_3 (Flatten) (None, 50176) 0
dense_7 (Dense) (None, 128) 6422656
dense_8 (Dense) (None, 64) 8256
dense_9 (Dense) (None, 3) 195
=================================================================
Total params: 6,431,107
Trainable params: 6,431,107
Non-trainable params: 0
_________________________________________________________________
# Train the mask model
model_mask.fit(masks_train, y_train_one_hot, epochs=7, batch_size=30)
Epoch 1/7 145/145 [==============================] - 10s 64ms/step - loss: 1.7174 - accuracy: 0.4607 Epoch 2/7 145/145 [==============================] - 9s 64ms/step - loss: 0.9744 - accuracy: 0.5481 Epoch 3/7 145/145 [==============================] - 9s 65ms/step - loss: 0.9155 - accuracy: 0.5763 Epoch 4/7 145/145 [==============================] - 9s 64ms/step - loss: 0.8559 - accuracy: 0.6010 Epoch 5/7 145/145 [==============================] - 9s 64ms/step - loss: 0.8251 - accuracy: 0.6160 Epoch 6/7 145/145 [==============================] - 9s 65ms/step - loss: 0.7939 - accuracy: 0.6363 Epoch 7/7 145/145 [==============================] - 9s 65ms/step - loss: 0.7697 - accuracy: 0.6423
<keras.callbacks.History at 0x2573d14e640>
# Evaluate the mask model on test data
mask_loss, mask_accuracy = model_mask.evaluate(masks_test, y_test_one_hot)
print("Mask Model - Test Loss:", mask_loss)
print("Mask Model - Test Accuracy:", mask_accuracy)
34/34 [==============================] - 0s 5ms/step - loss: 0.9566 - accuracy: 0.5913 Mask Model - Test Loss: 0.9566269516944885 Mask Model - Test Accuracy: 0.5913284420967102
# Combine predictions from both models
combined_pred = ((model_image.predict(X_test) + model_mask.predict(masks_test))/2)
combined_labels = np.argmax(combined_pred, axis=1)
true_labels = np.argmax(y_test_one_hot, axis=1)
34/34 [==============================] - 0s 5ms/step 34/34 [==============================] - 0s 5ms/step
# Confusion Matrix
conf_matrix = confusion_matrix(true_labels, combined_labels)
plt.figure(figsize=(8, 6))
sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=classes, yticklabels=classes)
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()
auc_score = roc_auc_score(y_test_one_hot, combined_pred)
print("Combined AUC_score:", auc_score)
Combined AUC_score: 0.8397445972790241
# classification report
class_report = classification_report(truelabels, combinedlabels, target_names=classes)
print(class_report)
precision recall f1-score support
COVID-19 0.72 0.73 0.73 406
Non-COVID 0.73 0.72 0.73 328
Normal 0.72 0.71 0.72 350
accuracy 0.72 1084
macro avg 0.72 0.72 0.72 1084
weighted avg 0.72 0.72 0.72 1084